{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from lightgbm import LGBMClassifier\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('/home/luke/Desktop/kaggle/Home_Credit_Default_Risk')\n",
    "x_val = pd.read_csv('x_val.csv')\n",
    "y_val = pd.read_csv('y_val.csv')\n",
    "partial_y_train = pd.read_csv('partial_y_train.csv')\n",
    "partial_x_train = pd.read_csv('partial_x_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data shape:  (239999, 239)\n",
      "Training label shape:  (239999, 1)\n",
      "Validation data shape:  (67510, 239)\n",
      "Validation label shape:  (67510, 1)\n"
     ]
    }
   ],
   "source": [
    "print('Training data shape: ', partial_x_train.shape)\n",
    "print('Training label shape: ', partial_y_train.shape)\n",
    "print('Validation data shape: ', x_val.shape)\n",
    "print('Validation label shape: ', y_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>2.197455179324805385e-01</th>\n",
       "      <th>0.000000000000000000e+00</th>\n",
       "      <th>1.000000000000000000e+00</th>\n",
       "      <th>0.000000000000000000e+00.1</th>\n",
       "      <th>5.263157894736841813e-02</th>\n",
       "      <th>2.454384230388970047e-03</th>\n",
       "      <th>5.693483146067415679e-02</th>\n",
       "      <th>5.533520533520533413e-02</th>\n",
       "      <th>4.377104377104376609e-02</th>\n",
       "      <th>1.560552770777368613e-02</th>\n",
       "      <th>...</th>\n",
       "      <th>0.000000000000000000e+00.154</th>\n",
       "      <th>0.000000000000000000e+00.155</th>\n",
       "      <th>0.000000000000000000e+00.156</th>\n",
       "      <th>0.000000000000000000e+00.157</th>\n",
       "      <th>0.000000000000000000e+00.158</th>\n",
       "      <th>0.000000000000000000e+00.159</th>\n",
       "      <th>0.000000000000000000e+00.160</th>\n",
       "      <th>0.000000000000000000e+00.161</th>\n",
       "      <th>0.000000000000000000e+00.162</th>\n",
       "      <th>0.000000000000000000e+00.163</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.219751</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002089</td>\n",
       "      <td>0.126730</td>\n",
       "      <td>0.094068</td>\n",
       "      <td>0.108866</td>\n",
       "      <td>0.157398</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.219757</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>0.001166</td>\n",
       "      <td>0.056180</td>\n",
       "      <td>0.061495</td>\n",
       "      <td>0.057239</td>\n",
       "      <td>0.093190</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.219760</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.052632</td>\n",
       "      <td>0.000935</td>\n",
       "      <td>0.236725</td>\n",
       "      <td>0.158301</td>\n",
       "      <td>0.217733</td>\n",
       "      <td>0.134897</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.219762</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002281</td>\n",
       "      <td>0.388515</td>\n",
       "      <td>0.158915</td>\n",
       "      <td>0.346801</td>\n",
       "      <td>0.129705</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.219765</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001320</td>\n",
       "      <td>0.119333</td>\n",
       "      <td>0.098333</td>\n",
       "      <td>0.098765</td>\n",
       "      <td>0.309272</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 239 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   2.197455179324805385e-01  0.000000000000000000e+00  \\\n",
       "0                  0.219751                       0.0   \n",
       "1                  0.219757                       0.0   \n",
       "2                  0.219760                       0.0   \n",
       "3                  0.219762                       0.0   \n",
       "4                  0.219765                       0.0   \n",
       "\n",
       "   1.000000000000000000e+00  0.000000000000000000e+00.1  \\\n",
       "0                       0.0                         0.0   \n",
       "1                       0.0                         1.0   \n",
       "2                       0.0                         1.0   \n",
       "3                       1.0                         1.0   \n",
       "4                       1.0                         1.0   \n",
       "\n",
       "   5.263157894736841813e-02  2.454384230388970047e-03  \\\n",
       "0                  0.000000                  0.002089   \n",
       "1                  0.105263                  0.001166   \n",
       "2                  0.052632                  0.000935   \n",
       "3                  0.000000                  0.002281   \n",
       "4                  0.000000                  0.001320   \n",
       "\n",
       "   5.693483146067415679e-02  5.533520533520533413e-02  \\\n",
       "0                  0.126730                  0.094068   \n",
       "1                  0.056180                  0.061495   \n",
       "2                  0.236725                  0.158301   \n",
       "3                  0.388515                  0.158915   \n",
       "4                  0.119333                  0.098333   \n",
       "\n",
       "   4.377104377104376609e-02  1.560552770777368613e-02  \\\n",
       "0                  0.108866                  0.157398   \n",
       "1                  0.057239                  0.093190   \n",
       "2                  0.217733                  0.134897   \n",
       "3                  0.346801                  0.129705   \n",
       "4                  0.098765                  0.309272   \n",
       "\n",
       "               ...               0.000000000000000000e+00.154  \\\n",
       "0              ...                                        0.0   \n",
       "1              ...                                        0.0   \n",
       "2              ...                                        0.0   \n",
       "3              ...                                        0.0   \n",
       "4              ...                                        0.0   \n",
       "\n",
       "   0.000000000000000000e+00.155  0.000000000000000000e+00.156  \\\n",
       "0                           0.0                           0.0   \n",
       "1                           0.0                           0.0   \n",
       "2                           0.0                           0.0   \n",
       "3                           0.0                           0.0   \n",
       "4                           0.0                           0.0   \n",
       "\n",
       "   0.000000000000000000e+00.157  0.000000000000000000e+00.158  \\\n",
       "0                           0.0                           0.0   \n",
       "1                           0.0                           0.0   \n",
       "2                           0.0                           0.0   \n",
       "3                           0.0                           0.0   \n",
       "4                           0.0                           0.0   \n",
       "\n",
       "   0.000000000000000000e+00.159  0.000000000000000000e+00.160  \\\n",
       "0                           0.0                           0.0   \n",
       "1                           0.0                           0.0   \n",
       "2                           1.0                           0.0   \n",
       "3                           0.0                           0.0   \n",
       "4                           1.0                           0.0   \n",
       "\n",
       "   0.000000000000000000e+00.161  0.000000000000000000e+00.162  \\\n",
       "0                           0.0                           0.0   \n",
       "1                           0.0                           0.0   \n",
       "2                           0.0                           1.0   \n",
       "3                           0.0                           0.0   \n",
       "4                           0.0                           1.0   \n",
       "\n",
       "   0.000000000000000000e+00.163  \n",
       "0                           0.0  \n",
       "1                           0.0  \n",
       "2                           0.0  \n",
       "3                           0.0  \n",
       "4                           0.0  \n",
       "\n",
       "[5 rows x 239 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partial_x_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test = pd.read_csv('x_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_use = np.array(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(48743, 239)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test_use.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>2.197455179324805385e-01</th>\n",
       "      <th>0.000000000000000000e+00</th>\n",
       "      <th>1.000000000000000000e+00</th>\n",
       "      <th>0.000000000000000000e+00.1</th>\n",
       "      <th>5.263157894736841813e-02</th>\n",
       "      <th>2.454384230388970047e-03</th>\n",
       "      <th>5.693483146067415679e-02</th>\n",
       "      <th>5.533520533520533413e-02</th>\n",
       "      <th>4.377104377104376609e-02</th>\n",
       "      <th>1.560552770777368613e-02</th>\n",
       "      <th>...</th>\n",
       "      <th>0.000000000000000000e+00.154</th>\n",
       "      <th>0.000000000000000000e+00.155</th>\n",
       "      <th>0.000000000000000000e+00.156</th>\n",
       "      <th>0.000000000000000000e+00.157</th>\n",
       "      <th>0.000000000000000000e+00.158</th>\n",
       "      <th>0.000000000000000000e+00.159</th>\n",
       "      <th>0.000000000000000000e+00.160</th>\n",
       "      <th>0.000000000000000000e+00.161</th>\n",
       "      <th>0.000000000000000000e+00.162</th>\n",
       "      <th>0.000000000000000000e+00.163</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.219751</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002089</td>\n",
       "      <td>0.126730</td>\n",
       "      <td>0.094068</td>\n",
       "      <td>0.108866</td>\n",
       "      <td>0.157398</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.219757</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>0.001166</td>\n",
       "      <td>0.056180</td>\n",
       "      <td>0.061495</td>\n",
       "      <td>0.057239</td>\n",
       "      <td>0.093190</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.219760</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.052632</td>\n",
       "      <td>0.000935</td>\n",
       "      <td>0.236725</td>\n",
       "      <td>0.158301</td>\n",
       "      <td>0.217733</td>\n",
       "      <td>0.134897</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.219762</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002281</td>\n",
       "      <td>0.388515</td>\n",
       "      <td>0.158915</td>\n",
       "      <td>0.346801</td>\n",
       "      <td>0.129705</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.219765</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001320</td>\n",
       "      <td>0.119333</td>\n",
       "      <td>0.098333</td>\n",
       "      <td>0.098765</td>\n",
       "      <td>0.309272</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 239 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   2.197455179324805385e-01  0.000000000000000000e+00  \\\n",
       "0                  0.219751                       0.0   \n",
       "1                  0.219757                       0.0   \n",
       "2                  0.219760                       0.0   \n",
       "3                  0.219762                       0.0   \n",
       "4                  0.219765                       0.0   \n",
       "\n",
       "   1.000000000000000000e+00  0.000000000000000000e+00.1  \\\n",
       "0                       0.0                         0.0   \n",
       "1                       0.0                         1.0   \n",
       "2                       0.0                         1.0   \n",
       "3                       1.0                         1.0   \n",
       "4                       1.0                         1.0   \n",
       "\n",
       "   5.263157894736841813e-02  2.454384230388970047e-03  \\\n",
       "0                  0.000000                  0.002089   \n",
       "1                  0.105263                  0.001166   \n",
       "2                  0.052632                  0.000935   \n",
       "3                  0.000000                  0.002281   \n",
       "4                  0.000000                  0.001320   \n",
       "\n",
       "   5.693483146067415679e-02  5.533520533520533413e-02  \\\n",
       "0                  0.126730                  0.094068   \n",
       "1                  0.056180                  0.061495   \n",
       "2                  0.236725                  0.158301   \n",
       "3                  0.388515                  0.158915   \n",
       "4                  0.119333                  0.098333   \n",
       "\n",
       "   4.377104377104376609e-02  1.560552770777368613e-02  \\\n",
       "0                  0.108866                  0.157398   \n",
       "1                  0.057239                  0.093190   \n",
       "2                  0.217733                  0.134897   \n",
       "3                  0.346801                  0.129705   \n",
       "4                  0.098765                  0.309272   \n",
       "\n",
       "               ...               0.000000000000000000e+00.154  \\\n",
       "0              ...                                        0.0   \n",
       "1              ...                                        0.0   \n",
       "2              ...                                        0.0   \n",
       "3              ...                                        0.0   \n",
       "4              ...                                        0.0   \n",
       "\n",
       "   0.000000000000000000e+00.155  0.000000000000000000e+00.156  \\\n",
       "0                           0.0                           0.0   \n",
       "1                           0.0                           0.0   \n",
       "2                           0.0                           0.0   \n",
       "3                           0.0                           0.0   \n",
       "4                           0.0                           0.0   \n",
       "\n",
       "   0.000000000000000000e+00.157  0.000000000000000000e+00.158  \\\n",
       "0                           0.0                           0.0   \n",
       "1                           0.0                           0.0   \n",
       "2                           0.0                           0.0   \n",
       "3                           0.0                           0.0   \n",
       "4                           0.0                           0.0   \n",
       "\n",
       "   0.000000000000000000e+00.159  0.000000000000000000e+00.160  \\\n",
       "0                           0.0                           0.0   \n",
       "1                           0.0                           0.0   \n",
       "2                           1.0                           0.0   \n",
       "3                           0.0                           0.0   \n",
       "4                           1.0                           0.0   \n",
       "\n",
       "   0.000000000000000000e+00.161  0.000000000000000000e+00.162  \\\n",
       "0                           0.0                           0.0   \n",
       "1                           0.0                           0.0   \n",
       "2                           0.0                           1.0   \n",
       "3                           0.0                           0.0   \n",
       "4                           0.0                           1.0   \n",
       "\n",
       "   0.000000000000000000e+00.163  \n",
       "0                           0.0  \n",
       "1                           0.0  \n",
       "2                           0.0  \n",
       "3                           0.0  \n",
       "4                           0.0  \n",
       "\n",
       "[5 rows x 239 columns]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partial_x_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>-2.806993906034893627e-06</th>\n",
       "      <th>0.000000000000000000e+00</th>\n",
       "      <th>0.000000000000000000e+00.1</th>\n",
       "      <th>1.000000000000000000e+00</th>\n",
       "      <th>0.000000000000000000e+00.2</th>\n",
       "      <th>9.348203259945449956e-04</th>\n",
       "      <th>1.307865168539325829e-01</th>\n",
       "      <th>7.388557388557388805e-02</th>\n",
       "      <th>1.021324354657687922e-01</th>\n",
       "      <th>2.569996399789525876e-01</th>\n",
       "      <th>...</th>\n",
       "      <th>0.000000000000000000e+00.158</th>\n",
       "      <th>0.000000000000000000e+00.159</th>\n",
       "      <th>0.000000000000000000e+00.160</th>\n",
       "      <th>0.000000000000000000e+00.161</th>\n",
       "      <th>0.000000000000000000e+00.162</th>\n",
       "      <th>0.000000000000000000e+00.163</th>\n",
       "      <th>1.000000000000000000e+00.15</th>\n",
       "      <th>0.000000000000000000e+00.164</th>\n",
       "      <th>1.000000000000000000e+00.16</th>\n",
       "      <th>0.000000000000000000e+00.165</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000627</td>\n",
       "      <td>0.044387</td>\n",
       "      <td>0.061443</td>\n",
       "      <td>0.034792</td>\n",
       "      <td>0.491595</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000031</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001512</td>\n",
       "      <td>0.154373</td>\n",
       "      <td>0.265830</td>\n",
       "      <td>0.147026</td>\n",
       "      <td>0.260475</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000073</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>0.002474</td>\n",
       "      <td>0.382022</td>\n",
       "      <td>0.184872</td>\n",
       "      <td>0.382716</td>\n",
       "      <td>0.361433</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000101</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.052632</td>\n",
       "      <td>0.001320</td>\n",
       "      <td>0.144944</td>\n",
       "      <td>0.118761</td>\n",
       "      <td>0.145903</td>\n",
       "      <td>0.134897</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000112</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002089</td>\n",
       "      <td>0.228387</td>\n",
       "      <td>0.128642</td>\n",
       "      <td>0.191919</td>\n",
       "      <td>0.344429</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 239 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   -2.806993906034893627e-06  0.000000000000000000e+00  \\\n",
       "0                   0.000008                       0.0   \n",
       "1                   0.000031                       0.0   \n",
       "2                   0.000073                       0.0   \n",
       "3                   0.000101                       0.0   \n",
       "4                   0.000112                       0.0   \n",
       "\n",
       "   0.000000000000000000e+00.1  1.000000000000000000e+00  \\\n",
       "0                         0.0                       1.0   \n",
       "1                         1.0                       1.0   \n",
       "2                         0.0                       1.0   \n",
       "3                         1.0                       0.0   \n",
       "4                         1.0                       1.0   \n",
       "\n",
       "   0.000000000000000000e+00.2  9.348203259945449956e-04  \\\n",
       "0                    0.000000                  0.000627   \n",
       "1                    0.000000                  0.001512   \n",
       "2                    0.105263                  0.002474   \n",
       "3                    0.052632                  0.001320   \n",
       "4                    0.000000                  0.002089   \n",
       "\n",
       "   1.307865168539325829e-01  7.388557388557388805e-02  \\\n",
       "0                  0.044387                  0.061443   \n",
       "1                  0.154373                  0.265830   \n",
       "2                  0.382022                  0.184872   \n",
       "3                  0.144944                  0.118761   \n",
       "4                  0.228387                  0.128642   \n",
       "\n",
       "   1.021324354657687922e-01  2.569996399789525876e-01  \\\n",
       "0                  0.034792                  0.491595   \n",
       "1                  0.147026                  0.260475   \n",
       "2                  0.382716                  0.361433   \n",
       "3                  0.145903                  0.134897   \n",
       "4                  0.191919                  0.344429   \n",
       "\n",
       "               ...               0.000000000000000000e+00.158  \\\n",
       "0              ...                                        0.0   \n",
       "1              ...                                        0.0   \n",
       "2              ...                                        0.0   \n",
       "3              ...                                        0.0   \n",
       "4              ...                                        0.0   \n",
       "\n",
       "   0.000000000000000000e+00.159  0.000000000000000000e+00.160  \\\n",
       "0                           0.0                           0.0   \n",
       "1                           0.0                           0.0   \n",
       "2                           0.0                           0.0   \n",
       "3                           0.0                           0.0   \n",
       "4                           1.0                           0.0   \n",
       "\n",
       "   0.000000000000000000e+00.161  0.000000000000000000e+00.162  \\\n",
       "0                           0.0                           0.0   \n",
       "1                           0.0                           0.0   \n",
       "2                           0.0                           0.0   \n",
       "3                           0.0                           0.0   \n",
       "4                           0.0                           0.0   \n",
       "\n",
       "   0.000000000000000000e+00.163  1.000000000000000000e+00.15  \\\n",
       "0                           0.0                          0.0   \n",
       "1                           0.0                          0.0   \n",
       "2                           1.0                          0.0   \n",
       "3                           0.0                          0.0   \n",
       "4                           0.0                          0.0   \n",
       "\n",
       "   0.000000000000000000e+00.164  1.000000000000000000e+00.16  \\\n",
       "0                           0.0                          0.0   \n",
       "1                           0.0                          0.0   \n",
       "2                           0.0                          1.0   \n",
       "3                           0.0                          0.0   \n",
       "4                           0.0                          1.0   \n",
       "\n",
       "   0.000000000000000000e+00.165  \n",
       "0                           0.0  \n",
       "1                           0.0  \n",
       "2                           0.0  \n",
       "3                           0.0  \n",
       "4                           0.0  \n",
       "\n",
       "[5 rows x 239 columns]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format the training and testing data \n",
    "train = np.array(application_train.drop(columns = 'TARGET'))\n",
    "test = np.array(application_test)\n",
    "\n",
    "train_labels = np.array(train_labels).reshape((-1, ))\n",
    "\n",
    "# 10 fold cross validation\n",
    "folds = KFold(n_splits=5, shuffle=True, random_state=50)\n",
    "\n",
    "# Validation and test predictions\n",
    "valid_preds = np.zeros(train.shape[0])\n",
    "test_preds = np.zeros(test.shape[0])\n",
    "\n",
    "# Iterate through each fold\n",
    "for n_fold, (train_indices, valid_indices) in enumerate(folds.split(train)):\n",
    "    # Training data for the fold\n",
    "    train_fold, train_fold_labels = train[train_indices, :], train_labels[train_indices]\n",
    "    \n",
    "    # Validation data for the fold\n",
    "    valid_fold, valid_fold_labels = train[valid_indices, :], train_labels[valid_indices]\n",
    "    \n",
    "    # LightGBM classifier with hyperparameters\n",
    "    clf = LGBMClassifier(\n",
    "        n_estimators=10000,\n",
    "        learning_rate=0.1,\n",
    "        subsample=.8,\n",
    "        max_depth=7,\n",
    "        reg_alpha=.1,\n",
    "        reg_lambda=.1,\n",
    "        min_split_gain=.01,\n",
    "        min_child_weight=2\n",
    "    )\n",
    "    \n",
    "    # Fit on the training data, evaluate on the validation data\n",
    "    clf.fit(train_fold, train_fold_labels, \n",
    "            eval_set= [(train_fold, train_fold_labels), (valid_fold, valid_fold_labels)], \n",
    "            eval_metric='auc', early_stopping_rounds=100, verbose = False\n",
    "           )\n",
    "    \n",
    "    # Validation preditions\n",
    "    valid_preds[valid_indices] = clf.predict_proba(valid_fold, num_iteration=clf.best_iteration_)[:, 1]\n",
    "    \n",
    "    # Testing predictions\n",
    "    test_preds += clf.predict_proba(test, num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits\n",
    "    \n",
    "    # Display the performance for the current fold\n",
    "    print('Fold %d AUC : %0.6f' % (n_fold + 1, roc_auc_score(valid_fold_labels, valid_preds[valid_indices])))\n",
    "    \n",
    "    # Delete variables to free up memory\n",
    "    del clf, train_fold, train_fold_labels, valid_fold, valid_fold_labels\n",
    "    gc.collect()\n",
    "    \n",
    "\n",
    "# Make a submission dataframe\n",
    "#submission = application_test[['SK_ID_CURR']]\n",
    "#submission['TARGET'] = test_preds\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import regularizers\n",
    "from keras.wrappers.scikit_learn import KerasClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(239999, 239)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partial_x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(unit,epochs,batch_size):\n",
    "    model = models.Sequential()\n",
    "    model.add(layers.Dense(64, input_shape=(239,)))\n",
    "    model.add(layers.Dense(64, activation='relu'))\n",
    "    model.add(layers.Dense(1, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    model.fit(partial_x_train,partial_y_train, epochs=20, verbose=0,batch_size=100,validation_data=(x_val, y_val))\n",
    "    y_pred = model.predict_proba(x_test_use)\n",
    "    return roc_auc_score(y_val, model.predict_proba(x_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 15h 14min 40s, sys: 48min 57s, total: 16h 3min 37s\n",
      "Wall time: 11h 16min 20s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Function to create model, required for KerasClassifier\n",
    "def create_model(optimizer, init):\n",
    "    model = models.Sequential()\n",
    "    model.add(layers.Dense(64, kernel_initializer=init, input_shape=(239,)))\n",
    "    model.add(layers.Dense(64, kernel_initializer=init, activation='relu'))\n",
    "    model.add(layers.Dense(1, kernel_initializer=init, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy',optimizer=optimizer, metrics=['accuracy'])\n",
    "    return(model)\n",
    "model = KerasClassifier(build_fn=create_model, verbose=0)\n",
    "\n",
    "# grid search epochs, batch size and optimizer\n",
    "optimizers = ['rmsprop', 'adam']\n",
    "init = ['normal']\n",
    "epochs = [5,10,20,25,50,100,200]\n",
    "batches = [100,200,400,500,600,700]\n",
    "param_grid = dict(optimizer = optimizers, init = init,epochs = epochs,batch_size = batches)\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, scoring='roc_auc')\n",
    "grid_result = grid.fit(partial_x_train, partial_y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.742933 using {'batch_size': 400, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.723196 (0.005718) with: {'batch_size': 100, 'epochs': 5, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.741225 (0.003531) with: {'batch_size': 100, 'epochs': 5, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.722961 (0.005566) with: {'batch_size': 100, 'epochs': 10, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.741018 (0.003982) with: {'batch_size': 100, 'epochs': 10, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.719698 (0.005179) with: {'batch_size': 100, 'epochs': 20, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.739504 (0.002853) with: {'batch_size': 100, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.717166 (0.002190) with: {'batch_size': 100, 'epochs': 25, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.738858 (0.003820) with: {'batch_size': 100, 'epochs': 25, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.719218 (0.008245) with: {'batch_size': 100, 'epochs': 50, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.710701 (0.003231) with: {'batch_size': 100, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.708248 (0.008035) with: {'batch_size': 100, 'epochs': 100, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.679366 (0.003396) with: {'batch_size': 100, 'epochs': 100, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.697190 (0.006491) with: {'batch_size': 100, 'epochs': 200, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.644906 (0.006594) with: {'batch_size': 100, 'epochs': 200, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.731512 (0.005258) with: {'batch_size': 200, 'epochs': 5, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.740685 (0.003327) with: {'batch_size': 200, 'epochs': 5, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.732114 (0.004958) with: {'batch_size': 200, 'epochs': 10, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.742313 (0.003347) with: {'batch_size': 200, 'epochs': 10, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.726890 (0.002027) with: {'batch_size': 200, 'epochs': 20, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.741736 (0.003270) with: {'batch_size': 200, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.727504 (0.005662) with: {'batch_size': 200, 'epochs': 25, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.740142 (0.003687) with: {'batch_size': 200, 'epochs': 25, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.717527 (0.009044) with: {'batch_size': 200, 'epochs': 50, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.716046 (0.005749) with: {'batch_size': 200, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.712799 (0.007318) with: {'batch_size': 200, 'epochs': 100, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.675861 (0.005402) with: {'batch_size': 200, 'epochs': 100, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.701426 (0.008477) with: {'batch_size': 200, 'epochs': 200, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.646009 (0.005897) with: {'batch_size': 200, 'epochs': 200, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.737462 (0.004435) with: {'batch_size': 400, 'epochs': 5, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.741274 (0.003128) with: {'batch_size': 400, 'epochs': 5, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.737156 (0.004833) with: {'batch_size': 400, 'epochs': 10, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.742293 (0.003428) with: {'batch_size': 400, 'epochs': 10, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.735620 (0.005095) with: {'batch_size': 400, 'epochs': 20, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.742933 (0.003567) with: {'batch_size': 400, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.733702 (0.005225) with: {'batch_size': 400, 'epochs': 25, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.741259 (0.003642) with: {'batch_size': 400, 'epochs': 25, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.723255 (0.001678) with: {'batch_size': 400, 'epochs': 50, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.728658 (0.003467) with: {'batch_size': 400, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.693780 (0.007273) with: {'batch_size': 400, 'epochs': 100, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.688531 (0.005960) with: {'batch_size': 400, 'epochs': 100, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.662465 (0.002259) with: {'batch_size': 400, 'epochs': 200, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.648916 (0.003333) with: {'batch_size': 400, 'epochs': 200, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.738035 (0.004775) with: {'batch_size': 500, 'epochs': 5, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.741093 (0.003081) with: {'batch_size': 500, 'epochs': 5, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.739059 (0.004651) with: {'batch_size': 500, 'epochs': 10, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.742109 (0.003535) with: {'batch_size': 500, 'epochs': 10, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.736731 (0.004239) with: {'batch_size': 500, 'epochs': 20, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.742512 (0.003160) with: {'batch_size': 500, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.733753 (0.006510) with: {'batch_size': 500, 'epochs': 25, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.740577 (0.003584) with: {'batch_size': 500, 'epochs': 25, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.724406 (0.006204) with: {'batch_size': 500, 'epochs': 50, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.729408 (0.003621) with: {'batch_size': 500, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.701254 (0.013973) with: {'batch_size': 500, 'epochs': 100, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.694542 (0.002764) with: {'batch_size': 500, 'epochs': 100, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.665136 (0.001633) with: {'batch_size': 500, 'epochs': 200, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.661286 (0.004592) with: {'batch_size': 500, 'epochs': 200, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.737459 (0.005262) with: {'batch_size': 600, 'epochs': 5, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.740680 (0.003300) with: {'batch_size': 600, 'epochs': 5, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.739307 (0.004571) with: {'batch_size': 600, 'epochs': 10, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.740911 (0.003760) with: {'batch_size': 600, 'epochs': 10, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.737799 (0.005259) with: {'batch_size': 600, 'epochs': 20, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.742902 (0.004202) with: {'batch_size': 600, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.735748 (0.004220) with: {'batch_size': 600, 'epochs': 25, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.742676 (0.003728) with: {'batch_size': 600, 'epochs': 25, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.727315 (0.005827) with: {'batch_size': 600, 'epochs': 50, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.733717 (0.005496) with: {'batch_size': 600, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.704781 (0.003332) with: {'batch_size': 600, 'epochs': 100, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.698620 (0.008411) with: {'batch_size': 600, 'epochs': 100, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.659739 (0.005250) with: {'batch_size': 600, 'epochs': 200, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.663657 (0.007202) with: {'batch_size': 600, 'epochs': 200, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.739265 (0.004352) with: {'batch_size': 700, 'epochs': 5, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.740731 (0.003638) with: {'batch_size': 700, 'epochs': 5, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.739775 (0.004196) with: {'batch_size': 700, 'epochs': 10, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.741955 (0.003163) with: {'batch_size': 700, 'epochs': 10, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.738575 (0.004821) with: {'batch_size': 700, 'epochs': 20, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.742503 (0.003697) with: {'batch_size': 700, 'epochs': 20, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.736426 (0.005562) with: {'batch_size': 700, 'epochs': 25, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.742302 (0.003690) with: {'batch_size': 700, 'epochs': 25, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.726700 (0.004272) with: {'batch_size': 700, 'epochs': 50, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.733155 (0.005275) with: {'batch_size': 700, 'epochs': 50, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.703935 (0.010199) with: {'batch_size': 700, 'epochs': 100, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.703173 (0.007574) with: {'batch_size': 700, 'epochs': 100, 'init': 'normal', 'optimizer': 'adam'}\n",
      "0.682902 (0.007273) with: {'batch_size': 700, 'epochs': 200, 'init': 'normal', 'optimizer': 'rmsprop'}\n",
      "0.678235 (0.006853) with: {'batch_size': 700, 'epochs': 200, 'init': 'normal', 'optimizer': 'adam'}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "\tprint(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import regularizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Dense(64, kernel_initializer='normal',input_shape=(239,)))\n",
    "model.add(layers.Dense(64, kernel_initializer='normal',activation='relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model.fit(partial_x_train,partial_y_train,epochs = 20,batch_size=400,validation_data = (x_val,y_val),verbose = 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7409343871448655"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_val, model.predict_proba(x_val))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
